#! encoding: utf-8
import re
import os, sys
import yaml
import os.path as love
import codecs
import Queue
import threading
import urllib2
import urllib
import time




if __name__ == '__main__':

# Setup environ

    
    sys.path.append('/home/mik/django-rarebooks/rarebooks/')
    os.environ['DJANGO_SETTINGS_MODULE'] = "rarebooks.settings"

from django.db.models.loading import get_models
loaded_models = get_models()

from goodies import models

MEDIA_BOOKS_URL = '/home/mik/django-rarebooks/media/'

URL = "http://www.rarebooks.ru/ru/religion/index.php?id23="


PATTERNS = { 
    'author':      re.compile(u"<span class=\"author\">(.*?)</span>", re.DOTALL),
    'name':        re.compile(u"<span class=\"zag\">(.*?)</span>", re.DOTALL),
    'place':       re.compile(u"<span class=\"search\">.*?<br>(.*?)</span>", re.DOTALL),
    'description': re.compile(u"<!-- description -->(.*?)<table", re.DOTALL),
    'year' :       re.compile(u"альтернативно:</b></td>.*?<td>(.*?)</td>", re.DOTALL),
    'price':       re.compile(u"a='([0-9]{3,10})\.00", re.DOTALL)}

PICTURES = [
    re.compile(u"photo_win\('(/common/img/uploaded/books/.*?)',\s''\);\"><img.*?src=", re.DOTALL),
    'http://www.rarebooks.ru']




books_ids = range(10000)[9700:10000]

queue = Queue.Queue()

class ThreadUrl(threading.Thread):

    """Threaded Url Grab"""
    def __init__(self, queue):
        threading.Thread.__init__(self)
        self.queue = queue

    def run(self):
        while True:
            #grabs host from queue
            books = self.queue.get()
            self.clone(books)
            #grabs urls of hosts and prints first 1024 bytes of page

            #signals to queue job is done
            self.queue.task_done()

    def clone(self,elem_id):

        
        kuku = {}
        def fuck(i): 
            if i: 
                return max(i)
            return None
        chunk = urllib.urlopen(URL+str(elem_id)).read()
        chunk = chunk.decode('cp1251')
        for i,j in PATTERNS.items(): 
            kuku[i] = j.findall(chunk)

        b = models.Book(
            author      = fuck(kuku['author']),
            name        = fuck(kuku['name'  ]),
            place       = fuck(kuku['place']),
            description = fuck(kuku['description']),
            year        = fuck(kuku['year']),
            price       = fuck(kuku['price']))
    

       
        #d2 = json.load(open("text.txt"))    
        b.save()
        if PICTURES[0].findall(chunk): 

            for i,j in enumerate(PICTURES[0].findall(chunk)):
                egozo=str(elem_id)+str(i)+'.jpg'
                urllib.urlretrieve(PICTURES[1]+j, love.join(MEDIA_BOOKS_URL,egozo))
                models.Picture(image=egozo,book=b).save()
            return True


        

start = time.time()
def main():

  #spawn a pool of threads, and pass them queue instance 
    for i in range(10):
        t = ThreadUrl(queue)
        t.setDaemon(True)
        t.start()
      
   #populate queue with data   
    for i in books_ids:
        queue.put(i)
   
   #wait on the queue until everything has been processed     
        queue.join()

main()
print "Elapsed Time: %s" % (time.time() - start)



#def clone(self,elem_id):
#        path = love.join(love.curdir,str(elem_id))
#        print path
#        try:
#            os.mkdir( path, 0755 )
#        except:
#            pass    
#    #

#        zigmund = {}
#        chunk = urllib.urlopen(URL+str(elem_id)).read()
#        chunk = chunk.decode('cp1251')
#        for i,j in PATTERNS.items(): 
#            zigmund[i] = j.findall(chunk)    
#    #

#        yaml.dump(zigmund,open(love.join(path,'data'),'w'),encoding='utf-8', allow_unicode=True)
#        #d2 = json.load(open("text.txt"))    #

#        for i,j in enumerate(PICTURES[0].findall(chunk)):
#            urllib.urlretrieve(PICTURES[1]+j, love.join(path,str(i)+'.jpg'))
#        return True


#def populate_books(elem_id):
#    for i in Category.objects.all():
#        source = "http://rarebooks.ru"+i.legacy_url+"printable.php?print=1"
#        for j in main.findall(source):
#            try
#                cat = open() 
#                yaml.dump(zigmund,open(love.join(path,'data'),'w'),encoding='utf-8', allow_unicode=True)

    


      
